# coding=utf-8
"""
    @project: 15python_spider
    @Author：frank
    @file： 02_maoyan_film_csv.py
    @date：2023/12/26 20:32
"""

import pymysql
import random
import re
import time

import requests
from utils.UserAgent import user_agent_list


class MaoyanSpider(object):
    def __init__(self):
        self.url = 'https://www.maoyan.com/board/4?offset={}'
        self.ua_list = user_agent_list
        self.conn = pymysql.connect(host='localhost', port=3306, user='root', password='123456', database='maoyan_film',
                                    charset='utf8')
        self.cursor = self.conn.cursor()

    # 获取数据
    def get_page(self, url):
        # 每次使用随机的user-agent
        headers = {
            'User-Agent': random.choice(self.ua_list)
        }
        req = requests.get(url=url, headers=headers)
        req.encoding = 'utf-8'
        html_content = req.text
        # req = request.Request(url=url, headers=headers)
        # res = request.urlopen(req)
        # html_content = res.read().decode('utf-8')

        self.parse_page(html_content)

    def parse_page(self, html_content):
        pattern = re.compile(
            r'data-val="{movieId:\d+}">(\w+?)</a></p>.+?<p class="star">(.+?)</p>.+?<p class="releasetime">(.+?)</p>',
            re.S)

        film_list = pattern.findall(html_content)
        self.write_page(film_list)

    def write_page(self, film_list):
        film_list_rows = []
        for film in film_list:
            film_list_rows.append([film[0].strip(), film[1].strip(), film[2].strip()])
        self.cursor.executemany("insert into board_film(name,actors,release_time) values (%s,%s,%s)", film_list)
        self.conn.commit()

    def main(self):
        for i in range(10):
            url = self.url.format(i * 10)
            self.get_page(url)
            time.sleep(random.randint(1, 3))

        self.cursor.close()
        self.conn.close()


if __name__ == "__main__":
    start = time.time()
    spider = MaoyanSpider()
    spider.main()
    end = time.time()
    print('执行时间:%.2f' % (end - start))
